In [1]:
"""
General workflow for importing a new session
"""
subject = 'P01' # TODO: change this for each subject
verbose = True  # change this for debugging

import matplotlib
%matplotlib inline

from deepthought.datasets.openmiir.preprocessing.pipeline import Pipeline
settings = dict(debug=False, mne_log_level='Info', sfreq=64) # optional pipeline settings
pipeline = Pipeline(subject, settings)


Couldn't import dot_parser, loading of dot files will not be possible.
Using gpu device 0: Tesla C2075

Import or Load Raw


In [2]:
pipeline.import_and_process_metadata(verbose=verbose)
pipeline.load_raw(verbose=verbose)


Skipping existing /imaging/deepthought/datasets/mpi2015/eeg/mne/P01-raw.fif
Loading raw data for subject "P01" from /imaging/deepthought/datasets/mpi2015/eeg/mne/P01-raw.fif

In [3]:
print pipeline.raw.info


<Info | 18 non-empty fields
    bads : list | 0 items
    buffer_size_sec : numpy.float64 | 10.0
    ch_names : list | Fp1, AF7, AF3, F1, F3, F5, F7, FT7, FC5, FC3
    chs : list | 69 items (EEG: 64, STIM: 1, EOG: 4)
    comps : list | 0 items
    custom_ref_applied : bool | False
    events : list | 0 items
    file_id : dict | 4 items
    filename : str | /imaging/d.../P01-raw.fif
    highpass : float | 0.0
    hpi_meas : list | 0 items
    hpi_results : list | 0 items
    lowpass : float | 104.0
    meas_date : numpy.ndarray | 2015-01-28 12:39:57
    meas_id : dict | 4 items
    nchan : int | 69
    projs : list | Average EEG reference: off
    sfreq : float | 512.0
    acq_pars : NoneType
    acq_stim : NoneType
    ctf_head_t : NoneType
    description : NoneType
    dev_ctf_t : NoneType
    dev_head_t : NoneType
    dig : NoneType
    experimenter : NoneType
    hpi_subsystem : NoneType
    line_freq : NoneType
    proj_id : NoneType
    proj_name : NoneType
    subject_info : NoneType
>

Reject Bad Channels


In [4]:
# switch to interactive GUI mode to scroll through data
%matplotlib tk 
pipeline.plot_raw();
pass


scroll using cursor keys, click on channels to mark as "bad"

In [5]:
# switch back to inline mode
%matplotlib inline

In [6]:
# pipeline.reset_bad_channels() # use this to reset the channel list if needed

In [4]:
# TODO: change these value manually
# If you would rather apply the bandpass filter first, continue until it is applied 
# and then come back to this cell
#pipeline.mark_bad_channels(None, save_to_raw=False) # nothing to change, None will keep old values
pipeline.mark_bad_channels(['P8', 'P10', 'T8'], save_to_raw=True)


The following channels have been marked as bad: ['P8', 'P10', 'T8']
Updating bad channel information in: /imaging/deepthought/datasets/mpi2015/eeg/mne/P01-raw.fif

In [5]:
pipeline.print_bad_channels()


bad channels: ['P8', 'P10', 'T8']

In [6]:
pipeline.interpolate_bad_channels() # Note: this will overwrite data


WARNING (deepthought.datasets.mpi2015.preprocessing.pipeline): The following channels are interpolated: ['P8', 'P10', 'T8']. This overwrites the channel data. To undo this, the raw data needs to be reloaded.
Channel interpolation is currently only implemented for EEG. The MEG channels marked as bad will remain untouched.
Optimization terminated successfully.
         Current function value: 0.000000
         Iterations: 7
         Function evaluations: 289
Computing interpolation matrix from 61 sensor positions
Interpolating 3 sensors

Check and Merge Trials


In [7]:
pipeline.check_trial_events()


540 events found
Events id: [  11   12   13   14   21   22   23   24   31   32   33   34   41   42   43
   44  111  112  113  114  121  122  123  124  131  132  133  134  141  142
  143  144  211  212  213  214  221  222  223  224  231  232  233  234  241
  242  243  244 1000 1111 2001]
[[    512       0     121]
 [    520       0    1000]
 [   8141       0     122]
 ..., 
 [2459915       0    2001]
 [2467882       0     224]
 [2467925       0    2001]]
/usr/local/lib/python2.7/dist-packages/mne-0.9.dev-py2.7.egg/mne/viz/misc.py:461: UserWarning: More events than colors available. You should pass a list of unique colors.
  warnings.warn('More events than colors available. '
1st event at  [ 1.]
last event at  [ 4820.16601562]

In [8]:
pipeline.check_trial_audio_onset_merge(use_audio_onsets=True, verbose=None)


540 events found
Events id: [  11   12   13   14   21   22   23   24   31   32   33   34   41   42   43
   44  111  112  113  114  121  122  123  124  131  132  133  134  141  142
  143  144  211  212  213  214  221  222  223  224  231  232  233  234  241
  242  243  244 1000 1111 2001]

In [9]:
pipeline.merge_trial_and_audio_onsets()


540 events found
Events id: [  11   12   13   14   21   22   23   24   31   32   33   34   41   42   43
   44  111  112  113  114  121  122  123  124  131  132  133  134  141  142
  143  144  211  212  213  214  221  222  223  224  231  232  233  234  241
  242  243  244 1000 1111 2001]
360 events found
Events id: [  11   12   13   14   21   22   23   24   31   32   33   34   41   42   43
   44  111  112  113  114  121  122  123  124  131  132  133  134  141  142
  143  144  211  212  213  214  221  222  223  224  231  232  233  234  241
  242  243  244 1111 2001]

Check PSD and Channels


In [10]:
pipeline.check_psd()
pipeline.check_psd(fmax=35)
# line noise will probably be visible @ multiples of 60Hz



In [11]:
pipeline.check_channel(0)
# quite some drift and movement in the breaks


(2478166,)

Bandpass Filtering


In [12]:
pipeline.bandpass_filter()


Band-pass filtering from 0.5 - 30 Hz
[Parallel(n_jobs=4)]: Done   1 out of  64 | elapsed:    0.8s remaining:   50.9s
[Parallel(n_jobs=4)]: Done  14 out of  64 | elapsed:    2.8s remaining:   10.0s
[Parallel(n_jobs=4)]: Done  27 out of  64 | elapsed:    4.9s remaining:    6.7s
[Parallel(n_jobs=4)]: Done  40 out of  64 | elapsed:    6.9s remaining:    4.1s
[Parallel(n_jobs=4)]: Done  53 out of  64 | elapsed:    8.9s remaining:    1.8s
[Parallel(n_jobs=4)]: Done  64 out of  64 | elapsed:   10.5s finished

In [16]:
%matplotlib tk 
pipeline.plot_raw();
pass


scroll using cursor keys, click on channels to mark as "bad"

In [17]:
%matplotlib inline

In [13]:
pipeline.check_channel(0)
# looks like we got rid of that drift - nice!
# what's that? eyeblinks?


(2478166,)

In [14]:
## check PSD again - after bandpass, before down-sampling
pipeline.check_psd()
pipeline.check_psd(fmax=35)
# 60Hz is still visible


Beat and EOG Epoching


In [15]:
pipeline.generate_beat_events() # Note: this includes cue-beats !!!
pipeline.beat_epochs.average().plot();


Loading stimulus metadata from /imaging/deepthought/datasets/mpi2015/meta/Stimuli_Meta.v1.xlsx
/usr/local/lib/python2.7/dist-packages/xlrd/xlsx.py:246: PendingDeprecationWarning: This method will be removed in future versions.  Use 'tree.iter()' or 'list(tree.iter())' instead.
  for elem in self.tree.getiterator():
/usr/local/lib/python2.7/dist-packages/xlrd/xlsx.py:292: PendingDeprecationWarning: This method will be removed in future versions.  Use 'tree.iter()' or 'list(tree.iter())' instead.
  for elem in self.tree.getiterator():
{1: 6, 2: 6, 3: 8, 4: 8, 11: 6, 12: 6, 13: 8, 14: 8, 21: 6, 22: 6, 23: 4, 24: 8}
<Epochs  |  n_events : 8140 (all good), tmin : -0.2 (s), tmax : 0.8 (s), baseline : (None, 0)>

In [16]:
pipeline.find_eog_events()
# wow - that's a lot of blinking


EOG channel index for this subject is: [64 65 66 67]
Filtering the data to remove DC offset to help distinguish blinks from saccades
Now detecting blinks and generating corresponding events
Number of EOG events detected : 1568

Down-Sampling


In [17]:
pipeline.downsample()


        from doc:
        WARNING: The intended purpose of this function is primarily to speed
                up computations (e.g., projection calculation) when precise timing
                of events is not required, as downsampling raw data effectively
                jitters trigger timings. It is generally recommended not to epoch
                downsampled data, but instead epoch and then downsample, as epoching
                downsampled data jitters triggers.

        NOTE: event onset collisions will be reported as warnings
              in that case, it might be a good idea to pick either the trial onset or audio onset events
              and delete the other ones before downsampling
        
down-sampling raw and events stim channel ...
saving events for stim channel "STI 014" (#68)
360 events found
Events id: [  11   12   13   14   21   22   23   24   31   32   33   34   41   42   43
   44  111  112  113  114  121  122  123  124  131  132  133  134  141  142
  143  144  211  212  213  214  221  222  223  224  231  232  233  234  241
  242  243  244 1111 2001]
resampling 69 channels...
processing channel #0
processing channel #1
processing channel #2
processing channel #3
processing channel #4
processing channel #5
processing channel #6
processing channel #7
processing channel #8
processing channel #9
processing channel #10
processing channel #11
processing channel #12
processing channel #13
processing channel #14
processing channel #15
processing channel #16
processing channel #17
processing channel #18
processing channel #19
processing channel #20
processing channel #21
processing channel #22
processing channel #23
processing channel #24
processing channel #25
processing channel #26
processing channel #27
processing channel #28
processing channel #29
processing channel #30
processing channel #31
processing channel #32
processing channel #33
processing channel #34
processing channel #35
processing channel #36
processing channel #37
processing channel #38
processing channel #39
processing channel #40
processing channel #41
processing channel #42
processing channel #43
processing channel #44
processing channel #45
processing channel #46
processing channel #47
processing channel #48
processing channel #49
processing channel #50
processing channel #51
processing channel #52
processing channel #53
processing channel #54
processing channel #55
processing channel #56
processing channel #57
processing channel #58
processing channel #59
processing channel #60
processing channel #61
processing channel #62
processing channel #63
processing channel #64
processing channel #65
processing channel #66
processing channel #67
processing channel #68
data shape after resampling: (69, 309771)
restoring events for stim channel "STI 014" (#68)
down-sampling epochs ...
TODO: down-sampling events (not in stim channel) ...

In [18]:
## PSD after down-sampling:
pipeline.check_psd(fmax=35)
# looks less smooth than without down-sampling



In [19]:
# check events after after resampling -> should get the same result as above
pipeline.check_resampled_trial_events(plot=True, verbose=False)


360 events found
Events id: [  11   12   13   14   21   22   23   24   31   32   33   34   41   42   43
   44  111  112  113  114  121  122  123  124  131  132  133  134  141  142
  143  144  211  212  213  214  221  222  223  224  231  232  233  234  241
  242  243  244 1111 2001]
event onset jitter (min, mean, max): -0.013671875 -0.00725911458333 0.0

ICA


In [20]:
pipeline.compute_ica(verbose=True)


Fitting ICA to data using 61 channels. 
Please be patient, this may take some time
Inferring max_pca_components from picks.
Selection by explained variance: 61 components
computing Extended Infomax ICA

In [21]:
%matplotlib inline
pipeline.plot_ica_components() # static plot


/usr/local/lib/python2.7/dist-packages/matplotlib/colorbar.py:215: UserWarning: Use the colorbar set_ticks() method instead.
  warnings.warn("Use the colorbar set_ticks() method instead.")
/usr/local/lib/python2.7/dist-packages/mne-0.9.dev-py2.7.egg/mne/viz/utils.py:125: UserWarning: Matplotlib function 'tight_layout' is not supported by your backend: `module://IPython.kernel.zmq.pylab.backend_inline`. Skipping subpplot adjusment.
  warn(msg % case)

In [22]:
pipeline.inspect_source_psd(1) # plot the PSD of an IC source to look for alpha activity
pipeline.inspect_source_psd(2)



In [23]:
pipeline.find_eog_artifact_sources(plot=True, verbose=False)


suggested EOG artifact channels:  [0, 1, 3, 54]
EOG artifact component scores:  [ 0.53093529  0.09295009  0.3315735   0.15046224]

In [24]:
pipeline.auto_detect_artifact_components()


    Searching for artifacts...
    found 1 artifact by skewness
    found 1 artifact by kurtosis
    found 1 artifact by variance
Artifact indices found:
    6, 10, 16
Ready.
merging [[0, 1, 3, 54], [6, 10, 16]]

In [25]:
pipeline.select_artifact_sources(selection='auto')


suggested channels to reject (selection="auto"):  [0, 1, 3, 6, 10, 16, 54]
To change the component selection, specify select=[...] (component numbers) or select=N (top-N) and run this command again!
current selection: [0, 1, 3, 6, 10, 16, 54]

In [35]:
pipeline.exclude_ica_components(selection=[0, 1, 3,11]) # TODO: adapt selection list
# use cells below to decide, which components to reject, then come back to this cell


excluding ICA components:  [0, 1, 3, 11]

In [34]:
pipeline.inspect_ica_component(11, [0, 700])
pipeline.inspect_source_epochs(11, mode='beats', vmax=5)
pipeline.inspect_source_epochs(11, mode='eog', vmax=5)



In [28]:
for ic in pipeline.suggested_artifact_components:
# for ic in pipeline.ica.exclude:
    pipeline.inspect_ica_component(ic, [0, 700])
    pipeline.inspect_source_epochs(ic, mode='beats', vmax=5)
    pipeline.inspect_source_epochs(ic, mode='eog', vmax=5)



In [36]:
# enable this for interactive mode
# %matplotlib tk

# pipeline.plot_sources(mode='raw')   # this may take a while
# pipeline.plot_sources(mode='beats')
pipeline.plot_sources(mode='eog');


NOTE: Plotting in non-interactive mode.

In [37]:
# switch back to non-interactive inline mode
%matplotlib inline

In [38]:
pipeline.assess_unmixing_quality(verbose=False)


Assess impact on average EOG artifact:
Assess cleaning of EOG epochs:
Transforming to ICA space (61 components)
Zeroing out 4 ICA components
Inverse transforming to PCA space
Reconstructing sensor space signals from 61 PCA components
Assess impact on raw. Check the amplitudes do not change:
Transforming to ICA space (61 components)
Zeroing out 4 ICA components
Inverse transforming to PCA space
Reconstructing sensor space signals from 61 PCA components
Assess impact on evoked. Check the amplitudes do not change:
Transforming to ICA space (61 components)
Zeroing out 4 ICA components
Inverse transforming to PCA space
Reconstructing sensor space signals from 61 PCA components

In [39]:
pipeline.save_ica('100p_64c') # save for later


Writing ica solution to /imaging/deepthought/datasets/mpi2015/eeg/mne/P01-100p_64c-ica.fif...

Application Test


In [3]:
pipeline.load_ica('100p_64c')


Reading /imaging/deepthought/datasets/mpi2015/eeg/mne/P01-100p_64c-ica.fif ...
Isotrak not found
    Read a total of 1 projection items:
        Average EEG reference (1 x 64)  idle
Now restoring ICA solution ...
Ready.

In [47]:
raw = pipeline.ica.apply(pipeline.raw, exclude=(pipeline.ica.exclude), copy=False)
# Note: this is no longer necessary as of mne-python version 0.10-dev
# if len(raw.info['bads']) > 0:
#     raw.interpolate_bads() # interpolate bad channels afterwards as they are not processed by the ICA

%matplotlib tk 
pipeline.plot_raw();
pass


Transforming to ICA space (63 components)
Zeroing out 2 ICA components
Inverse transforming to PCA space
Reconstructing sensor space signals from 64 PCA components
scroll using cursor keys, click on channels to mark as "bad"